﻿Imports System.ComponentModel
Imports System.Net
Imports System.Text.RegularExpressions
Imports System.Web
Imports System.Threading
Imports Co9320.My
Imports System.Globalization
Imports Microsoft.VisualBasic.CompilerServices
Imports <xmlns:nicolive="http://live.nicovideo.jp/">


Public Class Crawler

    Public Event WebExceptionThrown(ByVal sender As Object, ByVal e As WebExceptionThrownEventArgs)

    'Private WithEvents Service As New ServiceReference.ServiceSoapClient

    Public Sub New()
        StationWorker = New BackgroundWorker
        StationWorker.WorkerSupportsCancellation = True

        ProgramWorker = New BackgroundWorker
        'ProgramWorker.WorkerReportsProgress = True
        ProgramWorker.WorkerSupportsCancellation = True

        CrawlWorker = New BackgroundWorker
        CrawlWorker.WorkerReportsProgress = True
        CrawlWorker.WorkerSupportsCancellation = True
    End Sub

#Region "Station Getting"


    Private WithEvents StationWorker As BackgroundWorker
    Public Event StationGettingCompleted(ByVal sender As Object, ByVal e As StationCrawlingEventAges)

    Public Sub GetStationAsync(ByVal id As String)
        StationWorker.CancelAsync()
        StationWorker.RunWorkerAsync(id)
    End Sub

    Private Sub StationWorker_DoWork(ByVal sender As Object, ByVal e As DoWorkEventArgs) Handles StationWorker.DoWork
        Dim id = DirectCast(e.Argument, String).ToLower


        Dim staUri As Uri
        Dim iconUri As Uri
        Dim pattern As String

        If id.StartsWith("co") Then
            staUri = New Uri(String.Format(My.Settings.CommunityUriFormatForRetrieving, id))
            iconUri = New Uri(String.Format(My.Settings.CommunityIconUriFormat, id))
            pattern = My.Settings.CommunityPageRegEx

        ElseIf id.StartsWith("ch") Then
            staUri = New Uri(String.Format(My.Settings.ChannelUriFormatForRetrieving, id))
            iconUri = New Uri(String.Format(My.Settings.ChannelIconUriFormat, id))
            pattern = My.Settings.ChannelPageRegEx
        Else
            e.Result = Nothing
            Exit Sub
        End If

        Dim html = GetContent(staUri)

        Dim match = Regex.Match(html, pattern, RegexOptions.Singleline)
        If match.Success Then
            Dim station = New Station With { _
                .Id = id, _
                .Name = HttpUtility.UrlDecode(match.Groups("name").Value)}
            e.Result = station
        Else
            e.Result = Nothing
        End If

    End Sub

    Private Sub StationWorker_Completed(ByVal sender As Object, ByVal e As RunWorkerCompletedEventArgs) Handles StationWorker.RunWorkerCompleted

        If Not e.Cancelled Then
            RaiseEvent StationGettingCompleted(Me, New StationCrawlingEventAges(DirectCast(e.Result, Station)))
        End If

    End Sub

#End Region

#Region "Program Getting"
    Private WithEvents ProgramWorker As BackgroundWorker
    'Public Event ProgramGettingProgressChanged(ByVal sender As Object, ByVal e As LiveProgramsCrawlingEventAges)
    Public Event ProgramGettingCompleted(ByVal sender As Object, ByVal e As ProgramCrawlingEventAges)


    Public Sub GetProgramAsync(ByVal id As String)
        ProgramWorker.CancelAsync()
        ProgramWorker.RunWorkerAsync(id)
    End Sub

    Private Sub ProgramWorker_DoWork(ByVal sender As Object, ByVal e As DoWorkEventArgs) Handles ProgramWorker.DoWork

        System.Threading.Thread.Sleep(3000) ' 3秒るぅるぅ～

        Dim id = DirectCast(e.Argument, String).ToLower
        Dim html = GetContent(New Uri(String.Format(My.Settings.LiveProgramUriFormat, id)))


        Dim program = New LiveProgram With {.Id = id}

        Dim title As String = ""
        Dim content As String = ""

        If My.Settings.LiveProgramPage_ProgramRegEx <> "" Then
            ' TODO
        Else
            title = html.MatchedText(My.Settings.LiveProgramPage_TitleRegEx, "title").DecodeValue
            content = html.MatchedText(My.Settings.LiveProgramPage_ContentRegEx, "content").DecodeValue
        End If
        program.Title = title
        program.Content = content

        program.Caster = html.MatchedText(My.Settings.LiveProgramPage_CasterRegEx, "caster").DecodeValue
        'If program.Caster = "" Then
        '    program.Caster = "(Nothing)"
        'End If

        If title = "" AndAlso content = "" Then
            e.Result = Nothing
        Else
            e.Result = program
        End If

    End Sub

    'Private Sub ProgramWorker_ProgressChanged(ByVal sender As Object, ByVal e As ProgressChangedEventArgs) Handles ProgramWorker.ProgressChanged
    '    Dim content = DirectCast(e.UserState, String)

    '    RaiseEvent ProgramGettingProgressChanged(Me, )
    'End Sub

    Private Sub ProgramWorker_Completed(ByVal sender As Object, ByVal e As RunWorkerCompletedEventArgs) Handles ProgramWorker.RunWorkerCompleted
        Try
            If Not e.Cancelled Then
                RaiseEvent ProgramGettingCompleted(Me, New ProgramCrawlingEventAges(DirectCast(e.Result, LiveProgram)))
            End If
        Catch ex As Exception

        End Try
    End Sub


#End Region




#Region "Live Programs Crawling old"

    'Private ProgramsHighPriorityCrawlingCount As Integer
    'Private ProgramsNormalPriorityCrawlingCount As Integer
    'Private ProgramsLowPriorityCrawlingCount As Integer

    Private WithEvents CrawlWorker As BackgroundWorker
    Public Event CrawlingProgressChanged(ByVal sender As Object, ByVal e As LiveProgramsCrawlingEventAges)
    Public Event CrawlingCompleted(ByVal sender As Object, ByVal e As EventArgs)



    Public Sub CrawlLivePrograms(Optional ByVal preWaitSeconds As Integer = 55)


        CrawlWorker.CancelAsync()
        CrawlWorker.RunWorkerAsync(preWaitSeconds)

    End Sub

    Private counter As Integer
    Private recentId As String

    Private Sub CrawlWorker_DoWork(ByVal sender As Object, ByVal e As DoWorkEventArgs) Handles CrawlWorker.DoWork

        'System.Threading.Thread.Sleep(DirectCast(e.Argument, Integer) * 1000)



        'Dim currentPage = 1
        'Do
        '    Dim content = GetContent(New Uri(String.Format(My.Settings.LiveProgramsUriFormat, currentPage)))

        '    'Dim index = content.IndexOf("id=""search_result""")
        '    'If index < 0 Then
        '    '    Exit Do
        '    'Else
        '    '    content = content.Substring(index)
        '    'End If

        '    ' ページ数
        '    Dim urlPattern = My.Settings.LiveProgramsPageLinkRegEx
        '    maxPage = 0
        '    For Each m As Match In Regex.Matches(content, urlPattern, RegexOptions.Singleline)
        '        If m.Success Then
        '            Dim page = CInt(m.Groups(1).Value)
        '            SyncLock syncObject ' TODO Dell
        '                If maxPage < page Then
        '                    maxPage = page
        '                End If
        '            End SyncLock
        '        End If
        '    Next

        '    ' 1ページ分処理
        '    CrawlWorker.ReportProgress(0, content)

        '    currentPage += 1
        '    SyncLock syncObject
        '        If currentPage > maxPage OrElse maxPage <= 1 Then
        '            Exit Do
        '        End If
        '    End SyncLock
        '    If counter > 0 Then
        '        Exit Do
        '    End If
        '    System.Threading.Thread.Sleep(1500)

        'Loop
        'counter = (counter + 1) Mod 6

        Thread.Sleep(CInt((CInt(e.Argument) * &H3E8)))
        '        Dim str As String
        '        For Each str In MySettingsProperty.Settings.LiveProgramsUriFormats.Split(New Char() {","c}, StringSplitOptions.RemoveEmptyEntries)
        '            Dim uri As Uri
        '            Dim enumerator As IEnumerator
        '            Dim num As Integer = 1
        '            If Not uri.IsWellFormedUriString(String.Format(str, num), UriKind.Absolute) Then
        '                GoTo Label_018A
        '            End If
        'Label_0068:
        '            uri = New Uri(String.Format(str, num))
        '            Dim content As String = Me.GetContent(uri)
        '            Dim liveProgramsPageLinkRegEx As String = MySettingsProperty.Settings.LiveProgramsPageLinkRegEx
        '            Me.maxPage = 0
        '            Try
        '                enumerator = Regex.Matches(content, liveProgramsPageLinkRegEx, RegexOptions.Singleline).GetEnumerator
        '                Do While enumerator.MoveNext
        '                    Dim current As Match = DirectCast(enumerator.Current, Match)
        '                    If current.Success Then
        '                        Dim num2 As Integer = Conversions.ToInteger(current.Groups.Item(1).Value)
        '                        Dim expression As Object = Me.syncObject
        '                        ObjectFlowControl.CheckForSyncLockOnValueType(expression)
        '                        SyncLock expression
        '                            If (Me.maxPage < num2) Then
        '                                Me.maxPage = num2
        '                            End If
        '                            Continue Do
        '                        End SyncLock
        '                    End If
        '                Loop
        '            Finally
        '                If TypeOf enumerator Is IDisposable Then
        '                    TryCast(enumerator, IDisposable).Dispose()
        '                End If
        '            End Try
        '            Me.CrawlWorker.ReportProgress(0, content)
        '            num += 1
        '            Dim syncObject As Object = Me.syncObject
        '            ObjectFlowControl.CheckForSyncLockOnValueType(syncObject)
        '            SyncLock syncObject
        '                If ((num > Me.maxPage) OrElse (Me.maxPage <= 1)) Then
        '                    GoTo Label_018A
        '                End If
        '            End SyncLock
        '            If (Me.counter <= 0) Then
        '                Thread.Sleep(&H5DC)
        '                GoTo Label_0068
        '            End If
        'Label_018A:
        '        Next
        '        Me.counter = ((Me.counter + 1) Mod 6)

        'Dim scheduling = DirectCast(e.Argument, CrawlScheduling)
        Dim ids = New List(Of String)
        Dim hasError As Boolean

        'For Each category In My.Settings.LiveProgramCategories.Split(New Char() {","c}, StringSplitOptions.RemoveEmptyEntries)
        Dim currentPage = 1
        Dim prevId = recentId
        Dim startTime = Now
        Do

            Dim uri = New Uri(String.Format(CultureInfo.InstalledUICulture, My.Settings.LiveProgramsFeedUriFormat, currentPage))
            If Not uri.IsWellFormedOriginalString Then
                Exit Do
            End If

            Dim responseDate As DateTime
            Dim body = GetContent(uri, responseDate)

            If body = "" Then
                hasError = True
                Exit Do
            End If

            ' 生放送情報取得
            Dim result = New LiveProgramsResult With { _
                .CurrentPage = currentPage}

            ' ページ内生放送取得
            Dim programs = RetrieveLivePrograms(body, responseDate)
            For Each p In programs
                'If p.Station.Type = StationType.Community Then
                '    p.Category = category
                'End If
                result.LivePrograms.Add(p)
                ids.Add(p.Id)
            Next
            CrawlWorker.ReportProgress(0, result)

            ' MEMO
            If programs.Count > 0 AndAlso currentPage = 1 Then
                For i = 0 To programs.Count - 1
                    If programs(i).Station.Type = StationType.Channel Then
                        Continue For
                    End If
                    recentId = programs(i).Id
                    Exit For
                Next
            End If

            If programs.Count = 0 Then
                result.TotalPage = currentPage
                Exit Do
            End If

            If Me.counter = 0 Then
                ' 途中で全ページ取得あきらめ
                If startTime.AddSeconds(60) > Now Then
                    currentPage += 1
                Else
                    Exit Do
                End If
            Else
                Dim ps = From p In programs Where p.Id = prevId
                If ps.Count > 0 Then
                    Exit Do
                ElseIf currentPage < 3 Then
                    currentPage += 1
                Else
                    Exit Do
                End If
            End If

            System.Threading.Thread.Sleep(100)

        Loop
        'Next

        'Dim crawlingResult = New LiveProgramsCrawlingResult With { _
        '    .HasError = hasError, _
        '    .Scheduling = scheduling}
        'crawlingResult.Ids.AddRange(ids)
        'e.Result = crawlingResult



        Me.counter = (Me.counter + 1) Mod 6

    End Sub

    Private Shared Function RetrieveLivePrograms(ByVal body As String, ByVal responseDate As DateTime?) As IList(Of LiveProgram)

        Dim list = New List(Of LiveProgram)

        Try
            ' 不正なXMLのとりあえず対策
            body = body.Substring(0, body.IndexOf("</rss>") + 6)

            Dim doc = XDocument.Parse(body)

            For Each i In doc...<item>


                Dim content = i.<description>.Value.Replace(vbCrLf, " ").Replace(vbLf, "").Replace(vbCr, "")
                Dim title = i.<title>.Value.Replace(vbCrLf, " ").Replace(vbLf, "").Replace(vbCr, "")

                'Dim index = i.<author>.Value.IndexOf("/")

                Dim isFace As Boolean = False
                Dim category As String = ""
                For Each c In i...<category>
                    If c.Value = "顔出し" Then
                        isFace = True
                    Else
                        category = c.Value
                    End If
                Next

                Dim station = New Station With { _
                    .Id = i.<nicolive:community_id>.Value, _
                    .Name = i.<nicolive:community_name>.Value, _
                    .RecentAirdate = CDate(i.<pubDate>.Value), _
                    .RecentTitle = title}

                Dim program = New LiveProgram With { _
                    .Id = i.<guid>.Value, _
                    .Title = title, _
                    .Content = content, _
                    .StartDateTime = CDate(i.<pubDate>.Value), _
                    .VisitorCount = CInt(i.<nicolive:view>.Value), _
                    .CommentCount = CInt(i.<nicolive:num_res>.Value), _
                    .IsMemberOnly = CBool(i.<nicolive:member_only>.Value), _
                    .CrawlDateTime = If(responseDate, Now), _
                    .IsFace = isFace, _
                    .Category = category, _
                    .Station = station}

                If i.<nicolive:owner_name>.Value IsNot Nothing Then
                    program.Caster = i.<nicolive:owner_name>.Value
                End If

                list.Add(program)
            Next

        Catch ex As Exception
#If DEBUG Then
            Stop
#End If
        End Try

        Return list

    End Function

    Private maxPage As Integer
    Private syncObject As New Object

    Private Sub CrawlWorker_ProgressChanged(ByVal sender As Object, ByVal e As ProgressChangedEventArgs) Handles CrawlWorker.ProgressChanged

        Dim result = DirectCast(e.UserState, LiveProgramsResult)

        'Dim html = DirectCast(e.UserState, String)

        'Dim programs = New List(Of LiveProgram)
        'Dim crawlDateTime = Now

        'Dim pt = My.Settings.LiveProgramElementRegEx.Replace(vbLf, "").Replace(vbCr, "").Trim
        'For Each m As Match In Regex.Matches(html, pt, RegexOptions.Singleline)
        '    If Not m.Success Then
        '        Continue For
        '    End If

        '    Dim el = m.Value

        '    Dim id As String = ""
        '    Dim title As String = ""
        '    Dim content As String = ""

        '    Dim match As Match

        '    If My.Settings.LiveProgramElement_ProgramRegEx <> "" Then
        '        match = Regex.Match(el, My.Settings.LiveProgramElement_ProgramRegEx, RegexOptions.Singleline)
        '        If match.Success Then
        '            id = match.Groups("live_id").DecodeValue
        '            title = match.Groups("title").DecodeValue
        '            content = match.Groups("content").DecodeValue
        '        End If
        '    Else
        '        id = el.MatchedText(My.Settings.LiveProgramElement_ProgramIdRegEx, "live_id")
        '        title = el.MatchedText(My.Settings.LiveProgramElement_TitleRegEx, "title").DecodeValue
        '        content = el.MatchedText(My.Settings.LiveProgramElement_ContentRegEx, "content").DecodeValue
        '    End If


        '    Dim station = New Station With { _
        '        .Id = el.MatchedText(My.Settings.LiveProgramElement_StationIdRegEx, "sta_id"), _
        '        .Name = el.MatchedText(My.Settings.LiveProgramElement_StationNameRegEx, "sta_name").DecodeValue, _
        '        .IsOnAir = True}

        '    match = Regex.Match(el, My.Settings.LiveProgramElement_StartDateTimeRegEx, RegexOptions.Singleline)
        '    Dim startDateTime As DateTime
        '    If match.Success Then
        '        startDateTime = New DateTime(CInt(match.Groups("start_year").Value), _
        '                                     CInt(match.Groups("start_month").Value), _
        '                                     CInt(match.Groups("start_day").Value), _
        '                                     CInt(match.Groups("start_hour").Value), _
        '                                     CInt(match.Groups("start_minute").Value), 0)
        '    End If


        '    Dim program = New LiveProgram With { _
        '        .Id = id, _
        '        .Title = title, _
        '        .Content = content, _
        '        .StartDateTime = startDateTime, _
        '        .VisitorCount = CInt(el.MatchedText(My.Settings.LiveProgramElement_VisitorCountRegEx, "visitor_count")), _
        '        .CommentCount = CInt(el.MatchedText(My.Settings.LiveProgramElement_CommentCountRegEx, "comment_count")), _
        '        .IsMemberOnly = CBool(el.MatchedText(My.Settings.LiveProgramElement_MemberOnlyRegEx) <> ""), _
        '        .IsFace = CBool(el.MatchedText(My.Settings.LiveProgramElement_FaceRegEx) <> ""), _
        '        .Category = el.MatchedText(My.Settings.LiveProgramElement_TagRegEx, "tag").DecodeValue, _
        '        .CrawlDateTime = crawlDateTime, _
        '        .Station = station}

        '    station.RecentTitle = program.Title
        '    station.RecentAirdate = startDateTime

        '    If program.Id <> "" AndAlso station.Id <> "" Then
        '        programs.Add(program)
        '    End If

        'Next

        RaiseEvent CrawlingProgressChanged(Me, New LiveProgramsCrawlingEventAges(result.LivePrograms))

    End Sub

    Private Sub CrawlWorker_Completed(ByVal sender As Object, ByVal e As RunWorkerCompletedEventArgs) Handles CrawlWorker.RunWorkerCompleted
        If Not e.Cancelled Then
            RaiseEvent CrawlingCompleted(Me, New EventArgs)
        End If
    End Sub

#End Region

    Private Function GetContent(ByVal uri As Uri) As String

        Dim request = DirectCast(WebRequest.Create(uri), HttpWebRequest)

        'request.UserAgent = My.Settings.UserAgent

        If My.Settings.ProxyUserName <> "" OrElse _
           My.Settings.ProxyPassword <> "" Then
            request.Credentials = New NetworkCredential(My.Settings.ProxyUserName, My.Settings.ProxyPassword)
        End If

        Dim content As String = ""
        Try
            Dim response = request.GetResponse
            Using stream = response.GetResponseStream, _
                  reader = New System.IO.StreamReader(stream, System.Text.Encoding.UTF8)
                content = reader.ReadToEnd
            End Using
        Catch webEx As WebException
            RaiseEvent WebExceptionThrown(Me, New WebExceptionThrownEventArgs(webEx))
        End Try
        Return content

    End Function

    Private Shared Function GetContent(ByVal uri As Uri, ByRef [date] As DateTime) As String

        Dim request = DirectCast(WebRequest.Create(uri), HttpWebRequest)

        request.UserAgent = My.Settings.UserAgent

        If My.Settings.ProxyUserName <> "" OrElse _
           My.Settings.ProxyPassword <> "" Then
            request.Credentials = New NetworkCredential(My.Settings.ProxyUserName, My.Settings.ProxyPassword)
        End If

        Dim content As String = ""
        Try
            Dim response = DirectCast(request.GetResponse, HttpWebResponse)
            Using stream = response.GetResponseStream, _
                  reader = New System.IO.StreamReader(stream, System.Text.Encoding.UTF8)
                content = reader.ReadToEnd
            End Using

            Try
                If response.Headers(HttpResponseHeader.Date) IsNot Nothing Then
                    [date] = New DateTime(CDate(response.Headers(HttpResponseHeader.Date)).Ticks, DateTimeKind.Utc)
                Else
                    [date] = Now
                End If
            Catch ex As InvalidCastException
                [date] = Now
            End Try

        Catch webEx As WebException

        Catch ex As Exception

        End Try

        Return content
    End Function


End Class
